package com.chenerzhu.crawler.proxy.pool.job.crawler;

import com.chenerzhu.crawler.proxy.pool.entity.ProxyIp;
import com.chenerzhu.crawler.proxy.pool.entity.WebPage;
import com.chenerzhu.crawler.proxy.pool.job.JsoupUtil;
import com.chenerzhu.crawler.proxy.pool.job.crawler.AbstractCrawler;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.Date;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;

/**
 * @author chenerzhu
 * @create 2018-09-02 15:23
 * http://www.xicidaili.com
 **/
@Slf4j
public class XicidailiCrawlerJob extends AbstractCrawler {
    public XicidailiCrawlerJob(ConcurrentLinkedQueue<ProxyIp> proxyIpQueue, String pageUrl) {
        super(proxyIpQueue, pageUrl);
    }
    @Override
    public void parsePage(String url) {
        try {
            Document document = JsoupUtil.sendGetRequest(url, timeOut);
            Element element = document.select("#list > table > tbody").first();
            assert element != null;
            Elements tr = element.select("tr");
            for (Element row : tr) {
                String ip = row.select("td[data-title=IP]").first().text();
                String port = row.select("td[data-title=PORT]").first().text();
                String type = row.select("td[data-title=类型]").first().text();
                String location = row.select("td[data-title=位置]").first().text();
                ProxyIp proxyIp = new ProxyIp();
                proxyIp.setIp(ip);
                proxyIp.setPort(Integer.parseInt(port));
                proxyIp.setType(type);
                proxyIp.setCountry(location);
                proxyIp.setLocation(ip + ":" + Integer.parseInt(port));
                proxyIp.setValidateCount(0);
                proxyIp.setAvailableCount(0);
                proxyIp.setUnAvailableCount(0);
                proxyIp.setAvailableRate(0.0);
                proxyIp.setAvailable(true);
                proxyIp.setCreateTime(new Date());
                proxyIp.setLastValidateTime(new Date());
                log.info("获取的ip代理================>> {}-->>{}", ip, url);
                proxyIpQueue.offer(proxyIp);
            }
        }catch (Exception e){
            log.info("请求" + url + "异常,请检查！");
        }
    }
}